package us.codecraft.webmagic.pipeline;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.Mongo;

public class HTMLToJSON {
	Document doc;
	
	public HTMLToJSON (Document doc) {
		this.doc = doc;
	}
	
	public static void main(String[] args) throws IOException {
		BasicDBObject mongoDoc = new BasicDBObject();
		mongoDoc = new HTMLToJSON(Jsoup.parse(new File("/home/eric/tmp/index.html"),"utf8")).ElementsToJSON();

		Mongo mc = new Mongo();
		DB db = mc.getDB("test");
		DBCollection coll = db.getCollection("jtest");
		coll.drop();
		coll.insert(mongoDoc);
	}
		
	private BasicDBObject ElToJSON (Element el) {
		BasicDBObject bdb = new BasicDBObject();

		bdb.append("Text", el.ownText());

		for (Attribute at : el.attributes()) {
			bdb.append(at.getKey(),at.getValue());
		}
		
		return bdb;
	}
	
	public BasicDBObject ElementsToJSON () {
		HashSet<String> hs_tagNames = new HashSet<String>();
		BasicDBObject bdb = new BasicDBObject();
		List<BasicDBObject> x = new ArrayList<BasicDBObject>();
		
		for (Element el : doc.getAllElements()) {
			hs_tagNames.add(el.tagName());
		} 

		for (String s : hs_tagNames) {
			
			for (Element el :doc.getElementsByTag(s) ) {
				x.add(ElToJSON(el));
			}
			
			bdb.append(s, new ArrayList<BasicDBObject>(x));
			x.clear();
		}
		return bdb;
	}
	
	public BasicDBObject AToJSON () {
		BasicDBObject bdb = new BasicDBObject();
		List<BasicDBObject> x = new ArrayList<BasicDBObject>();

		for (Element el :doc.getAllElements() ) {
			if (el.tagName().equals("a")) {
				x.add(new BasicDBObject().append(el.attr("href"), el.ownText()));
			}
		}
		bdb.append("OutLinks", x);
		return bdb;
	}

}


